<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
  <meta name="generator" content=
  "HTML Tidy for HTML5 for Linux version 5.6.0" />
  <meta content="width=device-width, initial-scale=1, shrink-to-fit=no" name=
  "viewport" />
  <meta content="text/html; charset=utf-8" http-equiv="Content-Type" />
  <script type='text/javascript' src=
  'https://s3.amazonaws.com/mturk-public/externalHIT_v1.js'></script><!-- BOOTSTRAP CSS -->
  <link crossorigin="anonymous" href=
  "https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css"
  integrity=
  "sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm"
  rel="stylesheet" /><!-- HITPUB CSS -->
  <link href=
  "https://fonts.googleapis.com/css?family=Open+Sans:400,400i,700,700i" rel=
  "stylesheet" />
  <style id="hitpub_css" type="text/css">
  /*<![CDATA[*/
  /***********************************************
                        MOSAIC BOOSTRAP OVERWRITES
                        ***********************************************/
                        #hitinfo .card {
                            border-radius: 0;
                        }
                        #hitinfo .card:first-child {
                            border-bottom: 0;
                        }
                         #hitinfo button.btn-link {
                            color: #06486F;
                            text-decoration: none;
                         }
                        #hitinfo button.btn-link:hover {
                            text-decoration: none;
                        }
                        #hit ul.question-choice {
                            list-style-type: none;
                        }
                        /***********************************************
                        MOSAIC GENERAL STYLING
                        ***********************************************/
                        body {
                             font-family: "Open Sans", "Roboto", sans-serif;
                         }
                        textarea#feedback {
                            width: 100%
                        }
                        input#submitButton {
                            margin: auto;
                            display: block;
                            background-color: #2172a4;
                            color: #fff;
                            font-size: 1.125rem;
                            padding: .5rem 1rem;
                            cursor: pointer;
                            border-radius: 1rem;
                        }
                        input#submitButton:hover {
                            background-color: #06486F;
                        }
                        
                        /***********************************************
                        TOTTOSTYLE
                        ***********************************************/
                        
                        table { border-collapse: collapse; }
                        th, td {
                          word-wrap: break-word;
                          max-width: 100%;
                          font-family: "Trebuchet MS", Arial, Helvetica, sans-serif;
                          border-bottom: 1px solid #ddd;
                          padding: 5px;
                          text-align: center;
                        }
                        tr:hover {background: #f4f4f4;}
                        tr:hover .highlighted {background: repeating-linear-gradient(
                               45deg,
                               #ffff99,
                               #ffff99 10px,
                               #f4f4f4 10px,
                               #f4f4f4 20px
                             );}
                        .highlighted { background-color: #ffff99; }
                        .center {
                            margin-left: auto;
                            margin-right: auto; }
  /*]]>*/
  </style>
  <title></title>
  <style type="text/css">
  /*<![CDATA[*/
  small.c16 {min-height:70px;}
  label.c15 {color:green;}
  label.c14 {color:red;}
  label.c13 {font-size: 32px}
  fieldset.c12 {text-align:center; border: 1px solid #D55E00; margin: 20px; padding: 0 10px 10px; border-radius: 8px; padding-top: 10px; box-shadow: 0 0 3px #666;}
  fieldset.c11 {text-align:center; border: 1px solid #0072B2; margin: 20px; padding: 0 10px 10px; border-radius: 8px; padding-top: 10px; box-shadow: 0 0 3px #666;}
  strong.c10 {color:lightcoral;}
  strong.c9 {color:lightgreen}
  strong.c8 {color:green}
  fieldset.c7 {border: 1px solid #D55E00; margin: 20px; padding: 0 10px 10px; border-radius: 8px; padding-top: 10px; box-shadow: 0 0 3px #666;}
  fieldset.c6 {border: 1px solid #0072B2; margin: 20px; padding: 0 10px 10px; border-radius: 8px; padding-top: 10px; box-shadow: 0 0 3px #666;}
  strong.c5 {color:red;}
  strong.c4 {color:green;}
  span.c3 {color:green;}
  span.c2 {color:red;}
  span.c1 {text-decoration: underline}
  /*]]>*/
  </style>
</head>
<body>
  <div class="container" id="hit">
    <div class="col-8 offset-2 accordion" id="hitinfo">
      <!-- INSTRUCTIONS START -->
      <div class="card">
        <div class="card-header" id="instructionsHeading">
          <h5 class="mb-0"><button aria-controls="instructions" aria-expanded=
          "true" class="btn btn-link" data-target="#instructions" data-toggle=
          "collapse" type="button">Instructions (click to expand)</button></h5>
        </div>
        <div aria-labelledby="instructionsHeading" class="collapse card-body"
        data-parent="#hitinfo" id="instructions">
          <p>Thank you for your participation in this and other similar
          HITS!</p>
          <p><em>Please take a moment to familiarize yourself with this new HIT
          by reading the instructions/examples, because things have changed a
          bit.</em> Thanks again for your work!</p>
          <p>In this HIT you will be presented with a <span class=
          "key-term1">table</span> from Wikipedia, with a few cells highlighted
          in yellow. Along with the table, you will be provided some metadata
          like the title of the Wikipedia article/section. Based on this table,
          you will also be given a <span class="key-term2">system
          generation,</span> which aims to capture/summarize/describe the Your
          job is to rate the generation across 2 axes:</p>
          <ul>
            <li><strong>Fluency/Grammaticality:</strong> <em>Is the system's
            generation <span class="c1">grammatical, easy-to-read, and
            fluent?</span></em></li>
            <li><strong>Correctness/Specificity:</strong> <em>Does the
            generation <span class="c1">correctly describe a fact from the
            table</span>, and does <span class="c1">that fact come from the
            highlighted cells?</span></em></li>
          </ul>
          <p>You will be able to rate each of the three axes on a scale from 1
          to 5, with <span class="c2">1 being the lowest/worst</span> and
          <span class="c3">5 the highest/best</span>. The specific scales
          are:</p>
          <ul>
            <li>
              <strong>Fluency/Grammaticality:</strong>
              <ul>
                <li><strong class="c4">5/5 (excellent):</strong> The generation
                is grammatical and fluent.</li>
                <li>4/5 (good): The sentence largely makes sense, but there are
                some small grammar issues/out-of-place words that don't make
                for the best writing.</li>
                <li>3/5 (okay): The grammar is okay and it's possible to read,
                but it definitely doesn't sound like a human wrote it.</li>
                <li>2/5 (poor): Even though I can kind-of tell the meaning,
                it's difficult to read this unnatural sentence.</li>
                <li><strong class="c5">1/5 (terrible):</strong> The generation
                has severe errors in grammaticality/is almost or completely
                unreadable.</li>
              </ul>
            </li>
          </ul>
          <ul>
            <li>
              <strong>Correctness/Specificity:</strong>
              <ul>
                <li><strong class="c4">5/5 (correct and based on the
                highlighted cells):</strong> The generation correctly describes
                the information conveyed in the highlighted cells.</li>
                <li>4/5 (mostly reasonable): The generation mostly describes
                the information in the highlighted cells with only small
                deviations.</li>
                <li>3/5 (neutral): The generation is somewhat
                plausible/relevant, but it's not as specific to the highlighted
                cells or correct as it could be.</li>
                <li>2/5 (mostly unreasonable): I see why this could be
                generated given the table/cells, but it doesn't make much
                sense.</li>
                <li><strong class="c5">1/5
                (wrong/nonsense/irrelevant):</strong> The generation doesn't
                seem to apply to the cells/tables at all, or doesn't make any
                sense.</li>
              </ul>
            </li>
          </ul><br />
          <strong>Notes:</strong>
          <ul>
            <li><span class="c1"><em>For Fluency/Grammaticality, don't worry
            about correctness!</em></span> There can be grammatical sentences
            that do not describe the associated table, and vice versa (see the
            examples).</li>
            <li><span class="c1"><em>For Correctness/Specificity, consider both
            the correctness of the statement given the table, and also its
            specificity to the highlighted cells:</em></span> don't give 5/5 if
            the generation applies better to unhighlighted cells.</li>
            <li><span class="c1"><em>For Correctness/Specificity, it's okay if
            the generation references the metadata like the title</em></span>
            --- points should be deducted for "specificity" if there are other
            <span class="c1">table cells</span> that are referenced more
            directly.</li>
            <li>A handful of tables are quite large! Still apply the same
            rating criteria, even if the tables have many rows.</li>
          </ul>
        </div>
      </div><!-- INSTRUCTIONS END --><!-- EXAMPLES START -->
      <div class="card">
        <div class="card-header" id="examplesHeading">
          <h5 class="mb-0"><button aria-controls="examples" aria-expanded=
          "true" class="btn btn-link" data-target="#examples" data-toggle=
          "collapse" type="button">Examples (click to expand)</button></h5>
        </div>
        <div aria-labelledby="examplesHeading" class="collapse card-body"
        data-parent="#hitinfo" id="examples">
          <h3>Example 1:</h3>
          <div class="row col-12">
            <h5><span class="key-term1">Table from Wikipedia:</span></h5>
            <div class="col-12 align-text-center">
              <fieldset class="c6">
                <h2>2012 Chicago Bears season</h2><br />
                <b>Section Title</b>: Regular season<br />
                <b>Table Section Text</b>: Stats updated to the end of the
                season (Week 17).
                <table>
                  <tbody>
                    <tr>
                      <th colspan="1" rowspan="1">Category</th>
                      <th colspan="1" rowspan="1">Player(s)</th>
                      <th colspan="1" rowspan="1">Value</th>
                      <th colspan="1" rowspan="1">NFL Rank</th>
                      <th colspan="1" rowspan="1">NFC Rank</th>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Passing Yards</td>
                      <td colspan="1" rowspan="1">Jay Cutler</td>
                      <td colspan="1" rowspan="1">3,033 yards</td>
                      <td colspan="1" rowspan="1">24th</td>
                      <td colspan="1" rowspan="1">12th</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Passing Touchdowns</td>
                      <td colspan="1" rowspan="1">Jay Cutler</td>
                      <td colspan="1" rowspan="1">19 TDs</td>
                      <td colspan="1" rowspan="1">21st</td>
                      <td colspan="1" rowspan="1">11th</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Rushing Yards</td>
                      <td colspan="1" rowspan="1">Matt Forte</td>
                      <td colspan="1" rowspan="1">1,094 yards</td>
                      <td colspan="1" rowspan="1">T-12th</td>
                      <td colspan="1" rowspan="1">6th</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Rushing Touchdowns</td>
                      <td colspan="1" rowspan="1">Michael Bush/Matt Forte</td>
                      <td colspan="1" rowspan="1">5 TDs</td>
                      <td colspan="1" rowspan="1">T-20th</td>
                      <td colspan="1" rowspan="1">T-11th</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Receptions</td>
                      <td colspan="1" rowspan="1">Brandon Marshall</td>
                      <td colspan="1" rowspan="1">118 rec §</td>
                      <td colspan="1" rowspan="1">T-2nd</td>
                      <td colspan="1" rowspan="1">2nd</td>
                    </tr>
                    <tr>
                      <td class="highlighted" colspan="1" rowspan="1">Receiving
                      Yards</td>
                      <td class="highlighted" colspan="1" rowspan="1">Brandon
                      Marshall</td>
                      <td class="highlighted" colspan="1" rowspan="1">1,508
                      yards §</td>
                      <td colspan="1" rowspan="1">3rd</td>
                      <td colspan="1" rowspan="1">2nd</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Receiving Touchdowns</td>
                      <td colspan="1" rowspan="1">Brandon Marshall</td>
                      <td colspan="1" rowspan="1">11 TDs</td>
                      <td colspan="1" rowspan="1">T-4th</td>
                      <td colspan="1" rowspan="1">3rd</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Points</td>
                      <td colspan="1" rowspan="1">Robbie Gould</td>
                      <td colspan="1" rowspan="1">96 points</td>
                      <td colspan="1" rowspan="1">25th</td>
                      <td colspan="1" rowspan="1">13th</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Kickoff Return Yards</td>
                      <td colspan="1" rowspan="1">Devin Hester</td>
                      <td colspan="1" rowspan="1">295 yards</td>
                      <td colspan="1" rowspan="1">22nd</td>
                      <td colspan="1" rowspan="1">10th</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Punt Return Yards</td>
                      <td colspan="1" rowspan="1">Devin Hester</td>
                      <td colspan="1" rowspan="1">621 yards</td>
                      <td colspan="1" rowspan="1">32nd</td>
                      <td colspan="1" rowspan="1">6th</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Tackles (combined)</td>
                      <td colspan="1" rowspan="1">Lance Briggs</td>
                      <td colspan="1" rowspan="1">40 tackles</td>
                      <td colspan="1" rowspan="1">40th</td>
                      <td colspan="1" rowspan="1">T-22nd</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Sacks</td>
                      <td colspan="1" rowspan="1">Julius Peppers</td>
                      <td colspan="1" rowspan="1">11.5 sacks</td>
                      <td colspan="1" rowspan="1">T-9th</td>
                      <td colspan="1" rowspan="1">T-5th</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">Interceptions</td>
                      <td colspan="1" rowspan="1">Tim Jennings</td>
                      <td colspan="1" rowspan="1">9 INTs</td>
                      <td colspan="1" rowspan="1">1st</td>
                      <td colspan="1" rowspan="1">1st</td>
                    </tr>
                  </tbody>
                </table>
              </fieldset>
            </div>
          </div>
          <div class="row col-12">
            <h5><span class="key-term2">System's generation <span class=
            "c1">(rate this!)</span>:</span></h5>
            <div class="col-12 align-text-center">
              <fieldset class="c7">
                In 2012, Chicago Bears wide reciever Brandon Marshall had 1,508
                receiving yards.
              </fieldset>
            </div>
          </div>&nbsp;
          <ul>
            <li><strong class="c8">Fluency/Grammaticality: 5/5</strong>
            <em>Why?</em> The sentence is grammatically correct and easy to
            read.</li>
            <li><strong class="c8">Correctness/Specificity:: 5/5</strong>
            <em>Why?</em> The highlighted cells are completely and correctly
            captured by the generation.</li>
          </ul>&nbsp;
          <h3>Example 2:</h3>
          <div class="row col-12">
            <h5><span class="key-term1">Table from Wikipedia:</span></h5>
            <div class="col-12 align-text-center">
              <fieldset class="c6">
                <h4>Gerard Piqué</h4><br />
                <b>Section Title</b>: International goals<br />
                <b>Table Section Text</b>: <i>None</i>
                <table>
                  <tbody>
                    <tr>
                      <th colspan="1" rowspan="1">No.</th>
                      <th colspan="1" rowspan="1">Date</th>
                      <th colspan="1" rowspan="1">Venue</th>
                      <th colspan="1" rowspan="1">Cap</th>
                      <th colspan="1" rowspan="1">Opponent</th>
                      <th colspan="1" rowspan="1">Score</th>
                      <th colspan="1" rowspan="1">Result</th>
                      <th colspan="1" rowspan="1">Competition</th>
                    </tr>
                    <tr>
                      <th colspan="1" rowspan="1">1</th>
                      <td colspan="1" rowspan="1">28 March 2009</td>
                      <td colspan="1" rowspan="1">Santiago Bernabéu Stadium,
                      Madrid, Spain</td>
                      <td colspan="1" rowspan="1">2</td>
                      <td colspan="1" rowspan="1">Turkey</td>
                      <td colspan="1" rowspan="1">1–0</td>
                      <td colspan="1" rowspan="1">1–0</td>
                      <td colspan="1" rowspan="1">2010 FIFA World Cup
                      qualification</td>
                    </tr>
                    <tr>
                      <th colspan="1" rowspan="1">2</th>
                      <td colspan="1" rowspan="1">12 August 2009</td>
                      <td colspan="1" rowspan="1">Philip II Arena, Skopje,
                      Macedonia</td>
                      <td colspan="1" rowspan="1">8</td>
                      <td colspan="1" rowspan="1">North Macedonia</td>
                      <td colspan="1" rowspan="1">2–2</td>
                      <td colspan="1" rowspan="1">3–2</td>
                      <td colspan="1" rowspan="1">Friendly</td>
                    </tr>
                    <tr>
                      <th colspan="1" rowspan="1">3</th>
                      <td colspan="1" rowspan="1">5 September 2009</td>
                      <td colspan="1" rowspan="1">Estadio Riazor, A Coruña,
                      Spain</td>
                      <td colspan="1" rowspan="1">9</td>
                      <td colspan="1" rowspan="1">Belgium</td>
                      <td colspan="1" rowspan="1">3–0</td>
                      <td colspan="1" rowspan="1">5–0</td>
                      <td colspan="1" rowspan="2">2010 FIFA World Cup
                      qualification</td>
                    </tr>
                    <tr>
                      <th colspan="1" rowspan="1">4</th>
                      <td colspan="1" rowspan="1">14 October 2009</td>
                      <td colspan="1" rowspan="1">Bilino Polje, Zenica, Bosnia
                      and Herzegovina</td>
                      <td colspan="1" rowspan="1">12</td>
                      <td colspan="1" rowspan="1">Bosnia and Herzegovina</td>
                      <td colspan="1" rowspan="1">1–0</td>
                      <td colspan="1" rowspan="1">5–2</td>
                    </tr>
                    <tr>
                      <th class="highlighted" colspan="1" rowspan="1">5</th>
                      <td class="highlighted" colspan="1" rowspan="1">13 June
                      2016</td>
                      <td colspan="1" rowspan="1">Stadium Municipal, Toulouse,
                      France</td>
                      <td colspan="1" rowspan="1">78</td>
                      <td class="highlighted" colspan="1" rowspan="1">Czech
                      Republic</td>
                      <td class="highlighted" colspan="1" rowspan="1">1–0</td>
                      <td class="highlighted" colspan="1" rowspan="1">1–0</td>
                      <td class="highlighted" colspan="1" rowspan="1">UEFA Euro
                      2016</td>
                    </tr>
                  </tbody>
                </table>
              </fieldset>
            </div>
          </div>
          <div class="row col-12">
            <h5><span class="key-term2">System's generation <span class=
            "c1">(rate this!)</span>:</span></h5>
            <div class="col-12 align-text-center">
              <fieldset class="c7">
                Piqué scored an international goal at Stadium Municipal.
              </fieldset>
            </div>
          </div>
          <ul>
            <li><strong class="c9">Fluency/Grammaticality: 4/5</strong>
            <em>Why?</em> The sentence is grammatically correct, though the
            term "international goal" is a bit awkward --- "a goal on the
            international stage" would have been better.</li>
            <li><strong>Correctness/Specificity: 3/5</strong> <em>Why?</em> It
            might be correct, but the focus of the sentence is the stadium
            where the player has scored, but that cell is not highlighted.</li>
          </ul>&nbsp;
          <h3>Example 3:</h3>
          <div class="row col-12">
            <h5><span class="key-term1">Table from Wikipedia:</span></h5>
            <div class="col-12 align-text-center">
              <fieldset class="c6">
                <h4>1899 San Diego mayoral election</h4><br />
                <b>Section Title</b>: Election results<br />
                <b>Table Section Text</b>: <i>None</i>
                <table>
                  <tbody>
                    <tr>
                      <th colspan="2" rowspan="1">Party</th>
                      <th colspan="1" rowspan="1">Candidate</th>
                      <th colspan="1" rowspan="1">Votes</th>
                      <th colspan="1" rowspan="1">%</th>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">&nbsp;</td>
                      <td colspan="1" rowspan="1">Democratic</td>
                      <td colspan="1" rowspan="1">Edwin M. Capps</td>
                      <td colspan="1" rowspan="1">1,714</td>
                      <td colspan="1" rowspan="1">52.3</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">&nbsp;</td>
                      <td colspan="1" rowspan="1">Republican</td>
                      <td class="highlighted" colspan="1" rowspan="1">Daniel C.
                      Reed</td>
                      <td colspan="1" rowspan="1">1,493</td>
                      <td class="highlighted" colspan="1" rowspan="1">45.6</td>
                    </tr>
                    <tr>
                      <td colspan="1" rowspan="1">&nbsp;</td>
                      <td colspan="1" rowspan="1">Socialist Labor</td>
                      <td colspan="1" rowspan="1">John Helphingstine</td>
                      <td colspan="1" rowspan="1">70</td>
                      <td colspan="1" rowspan="1">2.1</td>
                    </tr>
                    <tr>
                      <th colspan="3" rowspan="1">Total votes</th>
                      <td colspan="1" rowspan="1">3,277</td>
                      <td colspan="1" rowspan="1">100</td>
                    </tr>
                  </tbody>
                </table>
              </fieldset>
            </div>
          </div>
          <div class="row col-12">
            <h5><span class="key-term2">System's generation <span class=
            "c1">(rate this!)</span>:</span></h5>
            <div class="col-12 align-text-center">
              <fieldset class="c7">
                Daniel C. Reed is a candidate 45.6 percent of the 1899 mayoral
                election results in San Diego, California. 45.6 --- Daniel C.
                Reed 1899 mayoral election result. cell
              </fieldset>
            </div>
          </div>
          <ul>
            <li><strong class="c10">Fluency/Grammaticality: 2/5</strong>
            <em>Why?</em> It's possible to fill-in-the-gaps to make sense of
            things, but this is not a fluent sentence.</li>
            <li><strong>Correctness/Specificity: 3/5</strong> <em>Why?</em> The
            sentence does seem to correctly copy the highlighted cells --- but
            it's not a correct description of the cells, it's just mindlessly
            copying.</li>
          </ul>
        </div>
      </div>
    </div><!-- MTURK INPUT START -->
    <div class="col-12 mt-5 offset-0 align-content-center">
      <form action="https://www.mturk.com/mturk/externalSubmit" id="mturk_form"
      method="post" name="mturk_form">
        <input id="assignmentId" name="assignmentId" type="hidden" value="" />
        <input id="ee" name="ee" type="hidden" />
        <div class="row col-12">
          <h5><span class="key-term1">Table from Wikipedia:</span></h5>
          <div class="col-12 align-text-center">
            <fieldset class="c11">
              ${prompt}
            </fieldset>
          </div>
        </div>
        <div class="row col-12">
          <h5><span class="key-term2">System's generation <span class=
          "c1">(rate this!)</span>:</span></h5>
          <div class="col-12 align-text-center">
            <fieldset class="c12">
              ${machine_completion}
            </fieldset>
          </div>
        </div>
        <div class=
        "row align-content-center mt-5 col-12 align-content-center form-group">
          <label for="coherence" id="coherencelabel" class=
          "c13"><strong>Coherence/Quality: 3/5</strong></label><br />
          <small>Is the system's generation <span class="c1">grammatical,
          easy-to-read, and fluent?</span></small><br />
          <label class="float-left c14">Bad</label> <label class=
          "float-right c15">Excellent</label><br />
          <input class="form-control-range" id="coherence" list=
          "coherence_list" max="5" min="1" name="coherence" step="1" type=
          "range" value="3" /> <datalist id="coherence_list">
            <option label="bad" value="1">
            </option>
            <option label="mediocre" value="2">
            </option>
            <option label="okay" value="3">
            </option>
            <option label="good" value="4">
            </option>
            <option label="excellent" value="5">
            </option>
          </datalist> <small class="form-text text-muted c16" id=
          "coherenceHelp">The grammar is okay and it's possible to read, but it
          definitely doesn't sound like a human wrote it.</small>
        </div>
        <div class=
        "row align-content-center col-12 align-content-center form-group">
          <label for="correct" id="correctlabel" class=
          "c13"><strong>Correctness/Specificity: 3/5</strong></label><br />
          <small>Does the generation <span class="c1">correctly describe a fact
          from the table</span>, and does <span class="c1">that fact come from
          the highlighted cells?</span></small><br />
          <label class="float-left c14">wrong/nonsense/irrelevant</label>
          <label class="float-right c15">correct and based on the highlighted
          cells</label><br />
          <input class="form-control-range" id="correct" list="correct_list"
          max="5" min="1" name="correct" step="1" type="range" value="3" />
          <datalist id="correct_list">
            <option label="bad" value="1">
            </option>
            <option label="mediocre" value="2">
            </option>
            <option label="okay" value="3">
            </option>
            <option label="good" value="4">
            </option>
            <option label="excellent" value="5">
            </option>
          </datalist> <small class="form-text text-muted c16" id=
          "correctHelp">The generation is somewhat plausible/relevant, but it's
          not as specific to the highlighted cells or correct as it could
          be.</small>
        </div><!-- OPTIONAL FEEDBACK -->
        <div class="row mt-5 col-8 offset-2 col-lg-6 offset-lg-3">
          <p>(Optional) Please let us know if anything was unclear, if you
          experienced any issues, or if you have any other feedback for us.</p>
          <textarea id="feedback" name="feedback" rows="3"></textarea>
        </div><!-- SUBMIT BUTTON -->
        <div class="row mt-5 col-2 offset-5">
          <input id="submitButton" onclick="getnext()" type="submit" value=
          "Submit" />
        </div>
      </form>
      <script language='Javascript'>
      <![CDATA[
      turkSetAssignmentID();
      ]]>
      </script>
    </div><!-- MTURK INPUT END -->
  </div><!-- HIT END --><!-- BOOSTRAP JS -->
  <script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity=
  "sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN"
  crossorigin="anonymous"></script> 
  <script src=
  "https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js"
  integrity=
  "sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q"
  crossorigin="anonymous"></script> 
  <script src=
  "https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js"
  integrity=
  "sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl"
  crossorigin="anonymous"></script><!-- HITPUB JS -->
   
  <script id="hitpub_js">


  <![CDATA[

                        
                        // TimeMe.js
                        (function () { var e, t; e = this, t = function () { var r = { startStopTimes: {}, idleTimeoutMs: 3e4, currentIdleTimeMs: 0, checkStateRateMs: 250, active: !1, idle: !1, currentPageName: "default-page-name", timeElapsedCallbacks: [], userLeftCallbacks: [], userReturnCallbacks: [], trackTimeOnElement: function (e) { var t = document.getElementById(e); t && (t.addEventListener("mouseover", function () { r.startTimer(e) }), t.addEventListener("mousemove", function () { r.startTimer(e) }), t.addEventListener("mouseleave", function () { r.stopTimer(e) }), t.addEventListener("keypress", function () { r.startTimer(e) }), t.addEventListener("focus", function () { r.startTimer(e) })) }, getTimeOnElementInSeconds: function (e) { var t = r.getTimeOnPageInSeconds(e); return t || 0 }, startTimer: function (e, t) { if (e || (e = r.currentPageName), void 0 === r.startStopTimes[e]) r.startStopTimes[e] = []; else { var n = r.startStopTimes[e], i = n[n.length - 1]; if (void 0 !== i && void 0 === i.stopTime) return } r.startStopTimes[e].push({ startTime: t || new Date, stopTime: void 0 }), r.active = !0, r.idle = !1 }, stopAllTimers: function () { for (var e = Object.keys(r.startStopTimes), t = 0; t < e.length; t++)r.stopTimer(e[t]) }, stopTimer: function (e, t) { e || (e = r.currentPageName); var n = r.startStopTimes[e]; void 0 !== n && 0 !== n.length && (void 0 === n[n.length - 1].stopTime && (n[n.length - 1].stopTime = t || new Date), r.active = !1) }, getTimeOnCurrentPageInSeconds: function () { return r.getTimeOnPageInSeconds(r.currentPageName) }, getTimeOnPageInSeconds: function (e) { var t = r.getTimeOnPageInMilliseconds(e); return void 0 === t ? void 0 : t / 1e3 }, getTimeOnCurrentPageInMilliseconds: function () { return r.getTimeOnPageInMilliseconds(r.currentPageName) }, getTimeOnPageInMilliseconds: function (e) { var t = r.startStopTimes[e]; if (void 0 !== t) { for (var n = 0, i = 0; i < t.length; i++) { var s = t[i].startTime, o = t[i].stopTime; void 0 === o && (o = new Date), n += o - s } return Number(n) } }, getTimeOnAllPagesInSeconds: function () { for (var e = [], t = Object.keys(r.startStopTimes), n = 0; n < t.length; n++) { var i = t[n], s = r.getTimeOnPageInSeconds(i); e.push({ pageName: i, timeOnPage: s }) } return e }, setIdleDurationInSeconds: function (e) { var t = parseFloat(e); if (!1 !== isNaN(t)) throw { name: "InvalidDurationException", message: "An invalid duration time (" + e + ") was provided." }; return r.idleTimeoutMs = 1e3 * e, this }, setCurrentPageName: function (e) { return r.currentPageName = e, this }, resetRecordedPageTime: function (e) { delete r.startStopTimes[e] }, resetAllRecordedPageTimes: function () { for (var e = Object.keys(r.startStopTimes), t = 0; t < e.length; t++)r.resetRecordedPageTime(e[t]) }, resetIdleCountdown: function () { r.idle && r.triggerUserHasReturned(), r.idle = !1, r.currentIdleTimeMs = 0 }, callWhenUserLeaves: function (e, t) { this.userLeftCallbacks.push({ callback: e, numberOfTimesToInvoke: t }) }, callWhenUserReturns: function (e, t) { this.userReturnCallbacks.push({ callback: e, numberOfTimesToInvoke: t }) }, triggerUserHasReturned: function () { if (!r.active) for (var e = 0; e < this.userReturnCallbacks.length; e++) { var t = this.userReturnCallbacks[e], n = t.numberOfTimesToInvoke; (isNaN(n) || void 0 === n || 0 < n) && (t.numberOfTimesToInvoke -= 1, t.callback()) } r.startTimer() }, triggerUserHasLeftPage: function () { if (r.active) for (var e = 0; e < this.userLeftCallbacks.length; e++) { var t = this.userLeftCallbacks[e], n = t.numberOfTimesToInvoke; (isNaN(n) || void 0 === n || 0 < n) && (t.numberOfTimesToInvoke -= 1, t.callback()) } r.stopAllTimers() }, callAfterTimeElapsedInSeconds: function (e, t) { r.timeElapsedCallbacks.push({ timeInSeconds: e, callback: t, pending: !0 }) }, checkState: function () { for (var e = 0; e < r.timeElapsedCallbacks.length; e++)r.timeElapsedCallbacks[e].pending && r.getTimeOnCurrentPageInSeconds() > r.timeElapsedCallbacks[e].timeInSeconds && (r.timeElapsedCallbacks[e].callback(), r.timeElapsedCallbacks[e].pending = !1); !1 === r.idle && r.currentIdleTimeMs > r.idleTimeoutMs ? (r.idle = !0, r.triggerUserHasLeftPage()) : r.currentIdleTimeMs += r.checkStateRateMs }, visibilityChangeEventName: void 0, hiddenPropName: void 0, listenForVisibilityEvents: function () { void 0 !== document.hidden ? (r.hiddenPropName = "hidden", r.visibilityChangeEventName = "visibilitychange") : void 0 !== document.mozHidden ? (r.hiddenPropName = "mozHidden", r.visibilityChangeEventName = "mozvisibilitychange") : void 0 !== document.msHidden ? (r.hiddenPropName = "msHidden", r.visibilityChangeEventName = "msvisibilitychange") : void 0 !== document.webkitHidden && (r.hiddenPropName = "webkitHidden", r.visibilityChangeEventName = "webkitvisibilitychange"), document.addEventListener(r.visibilityChangeEventName, function () { document[r.hiddenPropName] ? r.triggerUserHasLeftPage() : r.triggerUserHasReturned() }, !1), window.addEventListener("blur", function () { r.triggerUserHasLeftPage() }), window.addEventListener("focus", function () { r.triggerUserHasReturned() }), document.addEventListener("mousemove", function () { r.resetIdleCountdown() }), document.addEventListener("keyup", function () { r.resetIdleCountdown() }), document.addEventListener("touchstart", function () { r.resetIdleCountdown() }), window.addEventListener("scroll", function () { r.resetIdleCountdown() }), setInterval(function () { r.checkState() }, r.checkStateRateMs) }, websocket: void 0, websocketHost: void 0, setUpWebsocket: function (e) { if (window.WebSocket && e) { var t = e.websocketHost; try { r.websocket = new WebSocket(t), window.onbeforeunload = function () { r.sendCurrentTime(e.appId) }, r.websocket.onopen = function () { r.sendInitWsRequest(e.appId) }, r.websocket.onerror = function (e) { console && console.log("Error occurred in websocket connection: " + e) }, r.websocket.onmessage = function (e) { console && console.log(e.data) } } catch (e) { console && console.error("Failed to connect to websocket host.  Error:" + e) } } return this }, websocketSend: function (e) { r.websocket.send(JSON.stringify(e)) }, sendCurrentTime: function (e) { var t = { type: "INSERT_TIME", appId: e, timeOnPageMs: r.getTimeOnCurrentPageInMilliseconds(), pageName: r.currentPageName }; r.websocketSend(t) }, sendInitWsRequest: function (e) { var t = { type: "INIT", appId: e }; r.websocketSend(t) }, initialize: function (e) { var t = r.idleTimeoutMs || 30, n = r.currentPageName || "default-page-name", i = void 0, s = void 0; e && (t = e.idleTimeoutInSeconds || t, n = e.currentPageName || n, i = e.websocketOptions, s = e.initialStartTime), r.setIdleDurationInSeconds(t).setCurrentPageName(n).setUpWebsocket(i).listenForVisibilityEvents(), r.startTimer(void 0, s) } }; return r }, "undefined" != typeof module && module.exports ? module.exports = t() : "function" == typeof define && define.amd ? define([], function () { return e.TimeMe = t() }) : e.TimeMe = t() }).call(this);
                        
                        TimeMe.initialize({
                            currentPageName: "task",
                            idleTimeoutInSeconds: 30
                        });
                        
                        
                        
                        $(document).ready(function() {
                            var cookie_name = '_task_totto_RLUE_v1';
                            
                            $('.collapse').collapse({ 'toggle': false }).on('hidden.bs.collapse', function() {
                                if (this.id) {
                                    localStorage[this.id + cookie_name] = 'true';
                                }
                            }).on('shown.bs.collapse', function() {
                                if (this.id) {
                                    localStorage.removeItem(this.id + cookie_name);
                                }
                            }).each(function() {
                                if (this.id && localStorage[this.id + cookie_name] == 'true' ) {
                                    $(this).collapse('hide');
                                }
                            });
                        
                        
                            $('#submitButton').click(function () {
                                try {
                                    $('input[name=ee]').attr('value', TimeMe.getTimeOnCurrentPageInSeconds());
                                } catch {
                                }
                                return true;
                            });
                        
                            $('#coherence').on('input', function() {
                                var value = $(this).val()
                                if(value == 5) {
                                    $('#coherenceHelp').text(
                                        "The generation is grammatical and fluent.");
                                    $('#coherencelabel').html('<strong style="color:green;">Fluency/Grammaticality: 5/5</strong>');
                                }
                                else if(value == 4) {
                                    $('#coherenceHelp').text("The sentence largely makes sense, but there are some small grammar issues/out-of-place words that don't make for the best writing.");
                                    $('#coherencelabel').html('<strong style="color:mediumseagreen;">Fluency/Grammaticality: 4/5</strong>');
                                }
                                else if(value == 3) {
                                    $('#coherenceHelp').text("The grammar is okay and it's possible to read, but it definitely doesn't sound like a human wrote it.");
                                    $('#coherencelabel').html('<strong>Fluency/Grammaticality: 3/5</strong>');
                                }
                                else if(value == 2) {
                                    $('#coherenceHelp').text("Even though I can kind-of tell the meaning, it's difficult to read this unnatural sentence.");
                                    $('#coherencelabel').html('<strong style="color:lightcoral">Fluency/Grammaticality: 2/5</strong>');
                                }
                                else if(value == 1) {
                                    $('#coherenceHelp').text("The generation has severe errors in grammaticality/is almost or completely unreadable.");
                                    $('#coherencelabel').html('<strong style="color:red">Fluency/Grammaticality: 1/5</strong>');
                                }
                            });
                        
                        
                        
                            $('#correct').on('input', function() {
                                var value = $(this).val()
                                if(value == 5) {
                                    $('#correctHelp').text(
                                        "The generation correctly describes the information conveyed in the highlighted cells.");
                                    $('#correctlabel').html('<strong style="color:green;">Correctness/Specificity: 5/5</strong>');
                                }
                                else if(value == 4) {
                                    $('#correctHelp').text("The generation mostly describes the information in the highlighted cells with only small deviations.");
                                    $('#correctlabel').html('<strong style="color:mediumseagreen;">Correctness/Specificity: 4/5</strong>');
                                }
                                else if(value == 3) {
                                    $('#correctHelp').text("The generation is somewhat plausible/relevant, but it's not as specific to the highlighted cells or correct as it could be.");
                                    $('#correctlabel').html('<strong>Correctness/Specificity: 3/5</strong>');
                                }
                                else if(value == 2) {
                                    $('#correctHelp').text("I see why this could be generated given the table/cells, but it doesn't make much sense.");
                                    $('#correctlabel').html('<strong style="color:lightcoral">Correctness/Specificity: 2/5</strong>');
                                }
                                else if(value == 1) {
                                    $('#correctHelp').text("The generation doesn't seem to apply to the cells/tables at all, or doesn't make any sense.");
                                    $('#correctlabel').html('<strong style="color:red">Correctness/Specificity: 1/5</strong>');
                                }
                            });
                        
                            
                        });
  ]]>
  </script>
</body>
</html>
